# -*- coding: utf-8 -*-
from scrapy import Request, Spider
import base64, json, re, time
from util import woff_util
from bs4 import BeautifulSoup
from items import DpInfoItem
import settings


head = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Host': 'www.dianping.com',
    'Pragma': 'no-cache',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44',
    'X-Requested-With': 'XMLHttpRequest'
}

outer_cities = [2311,2312,2313,2314,2315,2316,2317,2318,2319,2322,2325,2326,2327,2328,2331,2332,2342,2343,2344,2345,2346,2347,2348,2349,2351,2352,2353,2354,2355,2356,2357,2358,2359,2360,2361,2362,2363,2364,2365,2366,2367,2369,2370,2371,2372,2373,2374,2375,2376,2377,2378,2379,2380,2381,2382,2383,2384,2385,2386,2387,2388,2390,2391,2392,2393,2394,2395,2396,2397,2398,2399,2400,2401,2402,2403,2404,2405,2412,2413,2414,2415,2416,2417,2418,2419,2420,2421,2422,2423,2424,2425,2426,2427,2428,2429,2430,2431,2432,2433,2434,2435,2436,2437,2438,2439,2440,2441,2442,2443,2444,2445,2446,2447,2448,2449,2450,2451,2452,2453,2454,2455,2456,2457,2458,2459,2460,2461,2462,2463,2464,2465,2466,2467,2468,2469,2470,2471,2481,2482,2483,2484,2485,2486,2487,2488,2489,2490,2493,2496,2500,2501,2505,3033,3034,3080,3082,3094,3095,3097,3100,3106,3108,3112,3116,3118,3122,3124,3125,3128,3149,3150,3151,3154,3156,3163,3164,3165,3178,3181,3183,3185,3186,3187,3188,3189,3190,3191,3192,3193,3194,3196,3198,3199,3201,3203,3205,3206,3207,3208,3209,3210,3211,3212,3213,3214,3216,3217,3218,3219,3221,3222,3223,3224,3225,3226,3228,3230,3231,3232,3233,3234,3235,3236,3237,3238,3239,3240,3241,3242,3244,3245,3249,3250,3251,3252,3253,3254,3255,3257,3258,3259,3262,3263,3264,3265,3266,3267,3268,3269,3270,3271,3272,3273,3275,3276,3277,3279,3280,3281,3282,3283,3284,3285,3286,3287,3288,3291,3294,3298,3299,3300,3301,3302,3303,3304,3305,3306,3309,3310,3311,3312,3313,3325,3326,3328,3330,3331,3333,3334,3335,3336,3337,3338,3339,3340,3341,3342,3343,3345,3346,3347,3348,3349,3350,3352,3353,3354,3355,3356,3359,3361,3362,3363,3375,3377,3378,3379,3380,3381,3382,3383,3384,3387,3388,3389,3390,3391,3392,3393,3394,3395,3396,3397,3398,3399,3400,3401,3402,3403,3405,3413,3414,3415,3416,3417,3418,3420,3421,3422,3423,3424,3425,3426,3428,3429,3430,3431,3432,3433,3434,3435,3436,3437,3438,3439,3440,3442,3443,3444,3446,3447,3448,3449,3450,3451,3452,3453,3455,3456,3457,3458,3459,3460,3462,3463,3464,3465,3466,3467,3468,3469,3470,3471,3472,3474,3475,3476,3477,3478,3479,3480,3481,3482,3483,3484,3485,3486,3487,3488,3489,3490,3491,3492,3494,3495,3496,3497,3498,3499,4003,4004,4005,4006,4007,4009,4010,4012,4013,4014,4015,4019,4020,4021,4022,4023,4024,4025,4026,4028,4029,4030,4031,4032,4034,4035,4036,4037,4038,4039,4040,4042,4043,4044,4045,4046,4047,4048,4049,4050,4051,4053,4054,4056,4057,4058,4059,4061,4062,4064,4065,4066,4069,4070,4071,4072,4074,4075,4077,4078,4079,4080,4082,4083,4084,4085,4086,4087,4088,4089,4090,4091,4092,4093,4094,4095,4097,4099,4100,4101,4102,4105,4107,4108,4110,4111,4113,4114,4119,4121,4122,4123,4124,4125,4126,4127,4128,4130,4131,4132,4133,4134,4135,4136,4137,4138,4139,4140,4141,4142,4143,4144,4146,4147,4148,4149,4150,4151,4152,4154,4156,4157,4159,4160,4161,4162,4163,4165,4166,4167,4168,4169,4170,4171,4178,4179,4180,4181,4182,4183,4185,4186,4187,4188,4189,4190,4191,4193,4194,4195,4200,4207,4214,4223,4225,4233,4234,4237,4239,4240,4242,4243,4249,4250,4255,4257,4258,4259,4262,4264,4268,4269,4271,4274,4278,4284,4285,4286,4287,4288,4294,4295,4296,4297,4298,4299,4300,4301,4302,4303,4304,4305,4306,4307,4308,4311,4312,4313,4314,4317,4318,4319,4320,4321,4322,4324,4326,4327,4330,4331,4332,4333,4336,4337,4338,4339,4340,4341,4342,4343,4344,4345,4347,4348,4349,4350,4351,4352,4353,4354,4356,4359,4360,4361,4362,4364,4365,4366,4367,4368,4369,4370,4371,4372,4373,4374,4376,4377,4378,4379,4380,4381,4382,4383,4384,4385,4388,4389,4390,4391,4392,4393,4394,4395,4396,4400,4401,4403,4405,4407,4409,4410,4411,4419,4420,4421,4422,4423,4446,4447,4448,4451,4452,7641,9278,10037,10049,10066,10079,10099,10122,10133,10163,10236,10243,10299,10348,10379,10410,10435,10485,10517,10520,10522,10604,10606,10662,10719,10725,10870,10901,10941,10988,10992,11097,11130,11161,11182,11281,11347,11434,11472,11518,11596,12205,12811,12867,13150,13854,13999,14430,15008,15009,15010,15011,15012,15013,15014,15015,15019,15020,15021,15022,15023,15024,15025,15028,15030,15032,15033,15034,15036,15039,15042,15043,15045,15046,15048,15051,15053,15056,15057,15069,15072,15073,15074,15075,15076,15077,15079,15088,15089,15093,15094,15097,15098,15099,15100,15101,15102,15105,15107,15111,15114,15116,15117,15119,15120,15121,15122,15124,15125,15129,15133,15134,15136,15138,15140,15141,15143,15144,15147,15149,15150,15152,15155,15156,15162,15163,15165,15166,15168,15170,15171,15175,15176,15177,15178,15180,15227,15228,15229,15234,15235,15236,15237,15239,15240,15241,15244,15245,15246,15247,15248,15250,15251,15252,15253,15254,15256,15257,15258,15259,15260,15261,16000,16001,16002,16003,16004,16005,16006,16007]

class DpInfoSpider(Spider):
    name = "dianping_info"

    def start_requests(self):
        # cookie = settings.COOKIES['dianping']
        # head["cookie"] = cookie
        # self.logger.info('cookie---' + cookie)
        urls = ["http://www.dianping.com/ajax/json/shopDynamic/basicHideInfo?shopId=%s&platform=1&partner=150&optimusCode=10" %i for i in range(100000000, 100000010)]
        for base_url in urls:
            req_url = base64.b64encode(base_url.encode()).decode('utf-8')
            request_map = {}
            request_map['url'] = req_url
            request_map['header'] = head
            url = 'http://spider.xiaodiankeji.net/http/v1/get?requestMap=' + str(request_map)
            yield Request(url=url, callback=self.parse)

    def parse(self, response):
        basic_body = response.body.decode('utf-8')
        self.logger.info(basic_body)
        info_obj = json.loads(basic_body)
        info_code = info_obj.get("code")
        if info_code == 500:
            self.logger.info("500-"+response.url)
            return
        shopInfo = info_obj.get("msg").get("shopInfo")
        city_id = shopInfo["cityId"]
        if city_id in outer_cities:
            self.logger.info("海外城市:::"+response.url)
            return
        kk = re.compile(r"<(\w+) class=\\\"(.*?)\\\">(.*?)<")
        cls_list = kk.findall(basic_body)
        for tag_s in cls_list:
            tag_name = tag_s[0]
            tag_type = tag_s[1]
            tag_code = tag_s[2]
            tag_ori = woff_util.get_decode(tag_code, tag_type)
            basic_body = basic_body.replace('<'+tag_name+' class=\\"'+tag_type+'\\">' + tag_code, '<'+tag_name+' class=\\"'+tag_type+'\\">' + tag_ori)
        info_obj = json.loads(basic_body)
        shopInfo = info_obj.get("msg").get("shopInfo")
        shop_id = shopInfo["shopId"]
        shop_type = shopInfo["shopType"]
        shop_name = shopInfo["shopName"]
        branch_name = shopInfo["branchName"]
        address_str = shopInfo["address"]
        address = ''
        if 'class=' in address_str:
            address = BeautifulSoup(address_str, "html.parser").text
        else:
            address = address_str
        cross_road_str = shopInfo["crossRoad"]
        cross_road = ''
        if cross_road_str is not None and 'class=' in cross_road_str:
            cross_road = BeautifulSoup(cross_road_str, "html.parser").text
        else:
            cross_road = cross_road_str
        phone_no_str = shopInfo["phoneNo"]
        phone_no = ''
        if 'class=' in phone_no_str:
            phone_no = BeautifulSoup(phone_no_str, "html.parser").text
        else:
            phone_no = phone_no_str
        phone_no2_str = shopInfo["phoneNo2"]
        phone_no2 = ''
        if 'class=' in phone_no2_str:
            phone_no2 = BeautifulSoup(phone_no2_str, "html.parser").text
        else:
            phone_no2 = phone_no2_str
        shop_group_id = shopInfo["shopGroupId"]
        alt_name = shopInfo["altName"]
        glat = shopInfo["glat"]
        glng = shopInfo["glng"]
        business_hours = shopInfo["businessHours"]
        gmt_time = int(time.time()*1000)
        yield DpInfoItem(shop_group_id=shop_group_id, shop_id=shop_id, shop_name=shop_name, shop_type=shop_type, city_id=city_id, address=address, 
            alt_name=alt_name, branch_name=branch_name, business_hours=business_hours, cross_road=cross_road, glat=glat, glng=glng, 
            phone_no=phone_no, phone_no2=phone_no2, gmt_create=gmt_time, gmt_update=gmt_time)
